#!/bin/bash
#SBATCH -J GCR                 #指定作业名称
#SBATCH -p kshdnormal
#SBATCH -N  4
#SBATCH --gres=dcu:4
#SBATCH --ntasks-per-node=4        #指定每节点的任务数量
#SBATCH --cpus-per-task=8           #指定每个任务使用CPU数量
#SBATCH --mem=24G          #指定每节点申请的内存大小。可以根据程序需要，自由指定。最大值为100GB。
##SBATCH --exclusive                 #指定独占节点
#SBATCH -o logs/runlog/scnet_%J.out
#SBATCH -e logs/runlog/scnet_%J.err
#SBATCH -t 00:02:00

echo "Start time: `date`"                               #显示开始时间
echo "SLURM_JOB_ID: $SLURM_JOB_ID"                      #显示作业号
echo "SLURM_NNODES: $SLURM_NNODES"                      #显示节点数
echo "SLURM_NTASKS: $SLURM_NTASKS"                      #显示总任务数
echo "SLURM_TASKS_PER_NODE: $SLURM_TASKS_PER_NODE"      #显示每节点的任务数
echo "SLURM_CPUS_PER_TASK: $SLURM_CPUS_PER_TASK"        #显示每个任务使用的CPU数量
echo "SLURM_JOB_PARTITION: $SLURM_JOB_PARTITION"        #显示队列/分区名称
echo "SLURM_SUBMIT_DIR:$SLURM_SUBMIT_DIR"               #显示提交作业目录的路径
echo "SLURM_NODELIST:$SLURM_NODELIST"                   #显示执行节点列表名称

srun hostname |sort |uniq -c |awk '{printf"%s slots=%s\n",$2,$1}' > .hostfile
node_num=${SLURM_JOB_NUM_NODES}
export NCCL_SOCKET_IFNAME=ib0
export NCCL_IB_HCA=mlx5_0
# if [ -e outfile_hip ]; then
#     cp outfile_hip{,_tmp}
#     mv outfile_hip "logs/runlog/outfile_hip.$(date +"%m-%d-%H%M-%S")"
# fi
# Define common MPI command options
# mpirun -np ${SLURM_NTASKS} --hostfile `pwd`/.hostfile \
#     --mca plm_rsh_no_tree_spawn 1 \
#     --mca plm_rsh_num_concurrent ${node_num} \
#     -mca routed_radix ${node_num} \
#     -mca pml ucx \
#     -x LD_LIBRARY_PATH \
#     -mca coll_hcoll_enable 0 \
#     --bind-to none `pwd`/scripts/job/dcu_bind.sh

LOGFILE="logs/results/scnet_$(date +"%m-%d-%H%M-%S").log"

# Run multiple test cases with nccl_flag from 1 to 7
# for nccl_flag in {9..9}; do
    echo "========================= nccl_flag=$nccl_flag =====================" >&2
    mpirun -np ${SLURM_NTASKS} --hostfile `pwd`/.hostfile \
        --mca plm_rsh_no_tree_spawn 1 \
    --mca plm_rsh_num_concurrent ${node_num} \
    -mca routed_radix ${node_num} \
    -mca pml ucx \
    -x LD_LIBRARY_PATH \
    -mca coll_hcoll_enable 0 \
     --bind-to none `pwd`/scripts/job/dcu_bind.sh
    # ./build/run.exe --resolution=0.25 --maxit=60 --nprocx=2  | tee -a $LOGFILE
    echo "" >> $LOGFILE  # Add blank line for readability
# done

echo "End time: `date`"  #显示结束时间
